package com.carl.main;

import com.carl.mapper.V2exMapper;
import com.carl.model.V2ex;
import com.carl.utils.MybatisHelper;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.print.attribute.standard.DocumentName;
import java.util.Date;

public class Day02_V2exCrawler {

    /** count_livid
            [\x{10000}-\x{10FFFF}] ==> "" // emjo 表情
        crawler-regex-tab :     #Tabs a
        crawler-regex-data :    div.cell.item
                crawler-regex-data[url] :    href
                crawler-regex-data[title] :    span.item_title
                crawler-regex-data[replyNum] :    a.count_livid
                crawler-regex-data[topicData] :    span.topic_info
     */
    // v2ex技术专区 : http://www.v2ex.com
    public static void main(String[] args)throws Exception{
         SqlSession sqlSession = MybatisHelper.getSqlSessionLocal();
         V2exMapper v2exMapper = sqlSession.getMapper(V2exMapper.class);
         String crawlerUrl = "http://www.v2ex.com" ;
         Document document = Jsoup.connect(crawlerUrl).get();

         Elements elements = document.select("#Tabs a");
         for (Element element : elements) {
             String hrefTab= element.attr("href");
             if ("/?tab=hot".equals(hrefTab) || "/?tab=all".equals(hrefTab)) {
                 continue;
             }
             String hrefUrl = crawlerUrl + hrefTab;
             System.out.println(" \t tab-href : " + hrefUrl);
             String dataType = element.text();

             Document typeDoc = Jsoup.connect(hrefUrl).get();
             Elements dataElementList = typeDoc.select("div.cell.item");
             for (Element dataElement : dataElementList) {
                 V2ex v2ex = new V2ex();
                 String dataUrl = crawlerUrl + dataElement.select("span.item_title a").attr("href");
                 String dataTitle = dataElement.select("span.item_title a").text().replaceAll("[\\x{10000}-\\x{10FFFF}]", "");
                 String dataReply = dataElement.select("a.count_livid").text();
                 Integer replyNum = null ;
                 try {
                     replyNum = Integer.parseInt(dataReply);
                 } catch (NumberFormatException e) {
                     replyNum = 0;
                 }

                 v2ex.setReplyNum(replyNum);
                 v2ex.setTitle(dataTitle);
                 v2ex.setUrl(dataUrl);

                 String[] topicDataArray = dataElement.select("span.topic_info").text().trim().replaceFirst("[1-9]","").trim().split(" ");
                 // 分享创造  •  shylockhg  •  1 分钟前  •  最后回复来自 glues
                 Integer topicDataArrayLength = topicDataArray.length;
                 if (topicDataArrayLength != 9 && topicDataArrayLength != 11) {
                     System.out.println(" \t\t exception--title : " + dataTitle);
                 } else {
                     String dataUser = topicDataArray[2];
                     String dataClazz = topicDataArray[0];
                     String upTime = topicDataArray[4] + topicDataArray[5];
                     if (topicDataArrayLength == 11) {
                         upTime = topicDataArray[4] + topicDataArray[5] + topicDataArray[6] + topicDataArray[7];
                     }
                     v2ex.setType(dataType);
                     v2ex.setUpTime(upTime);
                     v2ex.setUser(dataUser);
                     v2ex.setClazz(dataClazz);
                     v2ex.setCrawlerTime(new Date());

                     v2exMapper.insert(v2ex);
                 }
             }

         }
         sqlSession.commit(); // 提交事务
         sqlSession.close(); //  关闭链接
    }
    /**
      F12 : console.log($("#Main > div:nth-child(2) > div:nth-child(3) > table > tbody > tr > td:nth-child(3) > span.topic_info").text())
     */
}
